{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "excess-turkish",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "divine-rebate",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/mink/notebooks/CameraTraps')  # append this repo to PYTHONPATH\n",
    "sys.path.append('/home/mink/lib/ai4eutils')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "canadian-judge",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import Counter, defaultdict\n",
    "from random import sample\n",
    "from shutil import copyfile\n",
    "from multiprocessing.pool import ThreadPool\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "import path_utils, sas_blob_utils  # ai4eutils\n",
    "\n",
    "from data_management.megadb.schema import sequences_schema_check\n",
    "from data_management.annotations.add_bounding_boxes_to_megadb import *\n",
    "from data_management.megadb.converters.cct_to_megadb import process_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adapted-islam",
   "metadata": {},
   "source": [
    "# Adding images from existing megadb entries to annotation batch 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "gorgeous-spider",
   "metadata": {},
   "outputs": [],
   "source": [
    "def copy_file(src_path, dst_path):\n",
    "    return copyfile(src_path, dst_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "usual-southeast",
   "metadata": {},
   "source": [
    "## WCS Camera Traps\n",
    "\n",
    "Some are real sequences and others are dummy sequences with only one image. In Batch 9-11, we sampled sequences from every location with animal already."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "considered-anniversary",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/wcs_w_batch_10_boxes.json') as f:\n",
    "    wcs_db = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "quality-assignment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "407999"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wcs_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "equal-membership",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample(wcs_db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "direct-acoustic",
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_w_box = []\n",
    "num_im_w_box = 0\n",
    "\n",
    "seq_wo_box = []\n",
    "num_im_wo_box = 0\n",
    "\n",
    "for seq in wcs_db:\n",
    "    has_box = False\n",
    "    for im in seq['images']:\n",
    "        if 'bbox' in im:\n",
    "            has_box = True\n",
    "    if has_box:\n",
    "        seq_w_box.append(seq)\n",
    "        num_im_w_box += len(seq['images'])\n",
    "    else:\n",
    "        seq_wo_box.append(seq)\n",
    "        num_im_wo_box += len(seq['images'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "pregnant-footage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15897"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "61738"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "392102"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "1145752"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(seq_w_box)\n",
    "num_im_w_box\n",
    "\n",
    "len(seq_wo_box)\n",
    "num_im_wo_box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "breeding-leeds",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.9220763984881484"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1145752/392102"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "exciting-tongue",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3779"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_locations = set([seq['location'] for seq in wcs_db])\n",
    "len(all_locations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "leading-prophet",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3370"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations_w_box = set([seq['location'] for seq in seq_w_box])\n",
    "len(locations_w_box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "acoustic-transaction",
   "metadata": {},
   "outputs": [],
   "source": [
    "li_locations_w_box = [seq['location'] for seq in seq_w_box]\n",
    "counter_locations_w_box = Counter(li_locations_w_box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "coral-going",
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_wo_box_non_empty = []\n",
    "num_im_seq_wo_box_non_empty = 0\n",
    "\n",
    "for seq in seq_wo_box:\n",
    "    non_empty = False\n",
    "    for im in seq['images']:\n",
    "        if 'empty' not in im['class']:\n",
    "            non_empty = True\n",
    "            \n",
    "    if non_empty:\n",
    "        seq_wo_box_non_empty.append(seq)\n",
    "        num_im_seq_wo_box_non_empty += len(seq['images'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "endangered-bonus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "245069"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "639136"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(seq_wo_box_non_empty)\n",
    "num_im_seq_wo_box_non_empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "intimate-passenger",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.556544"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "639136 / 250000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "capable-massachusetts",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95730.078125"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "245069/2.56"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "seven-third",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "249091"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled = sample(seq_wo_box_non_empty, 137000)\n",
    "\n",
    "im_to_send = []\n",
    "\n",
    "for seq in sampled:\n",
    "    first_images = seq['images'][:7]\n",
    "    \n",
    "    for im in first_images:\n",
    "        im['seq_id'] = seq['seq_id']\n",
    "        \n",
    "        if 'frame_num' not in im:\n",
    "            im['frame_num'] = 1\n",
    "        \n",
    "        if 'empty' not in im['class']:\n",
    "            im_to_send.append(im)\n",
    "\n",
    "len(im_to_send)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "romantic-chess",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'datetime': '2010-01-28 17:03:37.000',\n",
       "  'frame_num': 1,\n",
       "  'match_level': 1,\n",
       "  'image_id': '3b9ddad6-92d5-11e9-ae26-000d3a74c7de',\n",
       "  'corrupt': False,\n",
       "  'wcs_id': 'ken-015-d0051-i022207',\n",
       "  'file': 'animals/0319/0153.jpg',\n",
       "  'sex': 'unknown',\n",
       "  'class': ['papio anubis'],\n",
       "  'count': 4,\n",
       "  'age': 'unknown',\n",
       "  'seq_id': 'ken-015-d0051-17'},\n",
       " {'datetime': '2009-03-29 11:57:00.000',\n",
       "  'match_level': 1,\n",
       "  'image_id': 'a6110c6c-92d5-11e9-952c-000d3a74c7de',\n",
       "  'corrupt': False,\n",
       "  'wcs_id': 'mdg-002-d0038-i002323',\n",
       "  'file': 'animals/0607/1244.jpg',\n",
       "  'sex': 'unknown',\n",
       "  'class': ['bos taurus'],\n",
       "  'count': 1,\n",
       "  'age': 'unknown',\n",
       "  'seq_id': 'dummy_7db87a086e7544a9bf22c8fa9d9ba8b5',\n",
       "  'frame_num': 1},\n",
       " {'datetime': '2013-05-24 07:49:45.000',\n",
       "  'match_level': 2,\n",
       "  'image_id': '89047789-92d5-11e9-9b52-000d3a74c7de',\n",
       "  'corrupt': False,\n",
       "  'wcs_id': 'gtm-003-d0015-i007368',\n",
       "  'file': 'animals/0529/1151.jpg',\n",
       "  'sex': 'unknown',\n",
       "  'class': ['motorcycle'],\n",
       "  'count': 1,\n",
       "  'age': 'unknown',\n",
       "  'seq_id': 'dummy_50efe9dfaf8f41c59790e8423851533b',\n",
       "  'frame_num': 1}]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(im_to_send, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "amber-garbage",
   "metadata": {},
   "outputs": [],
   "source": [
    "wcs_list_to_download = [im['file'] + '\\n' for im in im_to_send]\n",
    "\n",
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/wcs_files.txt', 'w') as f:\n",
    "    f.writelines(wcs_list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "decent-spyware",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 249091/249091 [00:01<00:00, 171603.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.35 s, sys: 103 ms, total: 1.45 s\n",
      "Wall time: 1.46 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'wcs'\n",
    "container_root = '/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for im in tqdm(im_to_send):\n",
    "    \n",
    "    src_path = os.path.join(container_root, im['file'])\n",
    "    \n",
    "    seq_id = im['seq_id']\n",
    "    frame = im['frame_num']\n",
    "\n",
    "    dst_path = os.path.join('/mink_disk_0/camtraps/imerit12e', \n",
    "                            f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "    path_pairs.append((src_path, dst_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "handed-finnish",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped/animals/0546/1370.jpg',\n",
       "  '/mink_disk_0/camtraps/imerit12e/wcs.seqdummy_7f6d7a9da5a34628a7a3157d2ea0e4f1.frame1.jpg'),\n",
       " ('/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped/animals/0117/1547.jpg',\n",
       "  '/mink_disk_0/camtraps/imerit12e/wcs.seqbol-016-d0062-27.frame6.jpg'),\n",
       " ('/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped/animals/0568/1379.jpg',\n",
       "  '/mink_disk_0/camtraps/imerit12e/wcs.seqdummy_15f43c4c29dc45f89d15499f00bf86d9.frame1.jpg'),\n",
       " ('/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped/animals/0143/1848.jpg',\n",
       "  '/mink_disk_0/camtraps/imerit12e/wcs.seqbol-015-d0047-56.frame2.jpg'),\n",
       " ('/mink_disk_0/camtraps/wcs-unzipped/wcs-unzipped/animals/0549/1822.jpg',\n",
       "  '/mink_disk_0/camtraps/imerit12e/wcs.seqdummy_d7f23faa29a14eedb744b94b87a04ad3.frame1.jpg')]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(path_pairs, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "elect-detail",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 49s, sys: 5min 22s, total: 7min 11s\n",
      "Wall time: 17min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(8) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "phantom-karen",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "249091"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "descending-coffee",
   "metadata": {},
   "source": [
    "## Snapshot Karoo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "buried-heath",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_karoo_megadb.json') as f:\n",
    "    karoo_db = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "objective-microwave",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14806"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_karoo',\n",
       "  'seq_id': 'KAR_S1#B03#1#610',\n",
       "  'location': 'B03',\n",
       "  'images': [{'id': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1692',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1692.JPG'},\n",
       "   {'id': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1693',\n",
       "    'frame_num': 2,\n",
       "    'file': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1693.JPG'},\n",
       "   {'id': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1694',\n",
       "    'frame_num': 3,\n",
       "    'file': 'KAR_S1/B03/B03_R1/KAR_S1_B03_R1_IMAG1694.JPG'}],\n",
       "  'class': ['gemsbokoryx'],\n",
       "  'datetime': '2018-01-29 00:24:23',\n",
       "  'resting': 0.0,\n",
       "  'standing': 0.56,\n",
       "  'moving': 0.11,\n",
       "  'young_present': 0.11,\n",
       "  'count': '1',\n",
       "  'season': 'KAR_S1',\n",
       "  'subject_id': 28807253,\n",
       "  'interacting': 0.0},\n",
       " {'dataset': 'snapshot_karoo',\n",
       "  'seq_id': 'KAR_S1#F02#1#1128',\n",
       "  'location': 'F02',\n",
       "  'images': [{'id': 'KAR_S1/F02/F02_R1/KAR_S1_F02_R1_IMAG2746',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KAR_S1/F02/F02_R1/KAR_S1_F02_R1_IMAG2746.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2017-10-21 22:02:34',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'KAR_S1',\n",
       "  'subject_id': 28741711,\n",
       "  'interacting': None}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(karoo_db)\n",
    "sample(karoo_db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "english-wheat",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(karoo_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "union-photograph",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2500"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in karoo_db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "anticipated-image",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6282"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'KAR/KAR_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "local-wisconsin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KAR/KAR_public/KAR_S1/A01/A01_R1/KAR_S1_A01_R1_IMAG00145.JPG\\n'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "verbal-implement",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_karoo_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "magnetic-torture",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 74.4 ms, sys: 57.2 ms, total: 132 ms\n",
      "Wall time: 1.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/E03/E03_R1/KAR_S1_E03_R1_IMAG0440.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#E03#1#168.frame1.jpg')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_karoo'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "related-floating",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/B02/B02_R1/KAR_S1_B02_R1_IMAG0230.JPG',\n",
       "  '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#B02#1#92.frame1.jpg'),\n",
       " ('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/B02/B02_R1/KAR_S1_B02_R1_IMAG0231.JPG',\n",
       "  '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#B02#1#92.frame2.jpg'),\n",
       " ('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/B02/B02_R1/KAR_S1_B02_R1_IMAG0232.JPG',\n",
       "  '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#B02#1#92.frame3.jpg'),\n",
       " ('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/B02/B02_R1/KAR_S1_B02_R1_IMAG0233.JPG',\n",
       "  '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#B02#1#93.frame1.jpg'),\n",
       " ('/mink_disk_0/camtraps/snapshot-safari/KAR/KAR_public/KAR_S1/B02/B02_R1/KAR_S1_B02_R1_IMAG0234.JPG',\n",
       "  '/mink_disk_0/camtraps/imerit12f/snapshot_karoo.seqKAR_S1#B02#1#93.frame2.jpg')]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_pairs[1000:1005]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "convertible-librarian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.2 s, sys: 40.8 s, total: 53.9 s\n",
      "Wall time: 43.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "funded-juvenile",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6282"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "transsexual-peeing",
   "metadata": {},
   "source": [
    "## Snapshot Kgalagadi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "alpine-swedish",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_kgalagadi_megadb.json') as f:\n",
    "    kgalagadi_db = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "adjacent-interference",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3566"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_kgalagadi',\n",
       "  'seq_id': 'KGA_S1#B06#1#897',\n",
       "  'location': 'B06',\n",
       "  'images': [{'id': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2683',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2683.JPG'},\n",
       "   {'id': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2684',\n",
       "    'frame_num': 2,\n",
       "    'file': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2684.JPG'},\n",
       "   {'id': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2685',\n",
       "    'frame_num': 3,\n",
       "    'file': 'KGA_S1/B06/B06_R1/KGA_S1_B06_R1_IMAG2685.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2018-12-22 13:20:18',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'KGA_S1',\n",
       "  'subject_id': 33332550,\n",
       "  'interacting': None},\n",
       " {'dataset': 'snapshot_kgalagadi',\n",
       "  'seq_id': 'KGA_S1#B09#1#166',\n",
       "  'location': 'B09',\n",
       "  'images': [{'id': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0488',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0488.JPG'},\n",
       "   {'id': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0489',\n",
       "    'frame_num': 2,\n",
       "    'file': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0489.JPG'},\n",
       "   {'id': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0490',\n",
       "    'frame_num': 3,\n",
       "    'file': 'KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0490.JPG'}],\n",
       "  'class': ['gemsbokoryx'],\n",
       "  'datetime': '2019-01-03 11:41:48',\n",
       "  'resting': 1.0,\n",
       "  'standing': 0.0,\n",
       "  'moving': 0.0,\n",
       "  'young_present': 0.0,\n",
       "  'count': 1.0,\n",
       "  'season': 'KGA_S1',\n",
       "  'subject_id': 33333260,\n",
       "  'interacting': 0.0}]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(kgalagadi_db)\n",
    "sample(kgalagadi_db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "lined-uganda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "894"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in kgalagadi_db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "involved-cheat",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2336"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'KGA/KGA_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "egyptian-corpus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KGA/KGA_public/KGA_S1/A01/A01_R1/KGA_S1_A01_R1_IMAG0230.JPG\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "athletic-butler",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_kgalagadi_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "intensive-stadium",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36.4 ms, sys: 16.1 ms, total: 52.5 ms\n",
      "Wall time: 614 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/KGA/KGA_public/KGA_S1/B09/B09_R1/KGA_S1_B09_R1_IMAG0586.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_kgalagadi.seqKGA_S1#B09#1#198.frame3.jpg')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_kgalagadi'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "supreme-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.54 s, sys: 13.7 s, total: 18.3 s\n",
      "Wall time: 8.79 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "partial-karen",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2336"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "funny-quantum",
   "metadata": {},
   "source": [
    "## Snapshot Enonkishu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "auburn-contractor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12969"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_enonkishu',\n",
       "  'seq_id': 'ENO_S1#D03#1#61',\n",
       "  'location': 'D03',\n",
       "  'images': [{'id': 'ENO_S1/D03/D03_R1/ENO_S1_D03_R1_IMAG0061',\n",
       "    'frame_num': 1,\n",
       "    'file': 'ENO_S1/D03/D03_R1/ENO_S1_D03_R1_IMAG0061.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2018-09-20 17:11:21',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'ENO_S1',\n",
       "  'subject_id': 31959195,\n",
       "  'interacting': None},\n",
       " {'dataset': 'snapshot_enonkishu',\n",
       "  'seq_id': 'ENO_S1#D04#2#288',\n",
       "  'location': 'D04',\n",
       "  'images': [{'id': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0710',\n",
       "    'frame_num': 1,\n",
       "    'file': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0710.JPG'},\n",
       "   {'id': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0711',\n",
       "    'frame_num': 2,\n",
       "    'file': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0711.JPG'},\n",
       "   {'id': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0712',\n",
       "    'frame_num': 3,\n",
       "    'file': 'ENO_S1/D04/D04_R2/ENO_S1_D04_R2_IMAG0712.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2018-11-15 11:20:29',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'ENO_S1',\n",
       "  'subject_id': 31959996,\n",
       "  'interacting': None}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_enonkishu_megadb.json') as f:\n",
    "    db = json.load(f)\n",
    "\n",
    "len(db)\n",
    "sample(db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "double-college",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4202"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "accurate-athletics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9496"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'ENO/ENO_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "postal-rates",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ENO/ENO_public/ENO_S1/B02/B02_R1/ENO_S1_B02_R1_IMAG0122.JPG\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "secondary-deficit",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_enonkishu_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "honey-philippines",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 99.6 ms, sys: 109 ms, total: 208 ms\n",
      "Wall time: 2.21 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/ENO/ENO_public/ENO_S1/F06/F06_R1/ENO_S1_F06_R1_IMAG0039.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_enonkishu.seqENO_S1#F06#1#14.frame2.jpg')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_enonkishu'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "needed-blackjack",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.8 s, sys: 1min 1s, total: 1min 19s\n",
      "Wall time: 1min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "married-stuff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9496"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stylish-alberta",
   "metadata": {},
   "source": [
    "## Snapshot Camdeboo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "turned-karen",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12024"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_camdeboo',\n",
       "  'seq_id': 'CDB_S1#C06#2#473',\n",
       "  'location': 'C06',\n",
       "  'images': [{'id': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1267',\n",
       "    'frame_num': 1,\n",
       "    'file': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1267.JPG'},\n",
       "   {'id': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1268',\n",
       "    'frame_num': 2,\n",
       "    'file': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1268.JPG'},\n",
       "   {'id': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1269',\n",
       "    'frame_num': 3,\n",
       "    'file': 'CDB_S1/C06/C06_R2/CDB_S1_C06_R2_IMAG1269.JPG'}],\n",
       "  'class': ['kudu'],\n",
       "  'resting': 0.0,\n",
       "  'standing': 0.1,\n",
       "  'moving': 0.9,\n",
       "  'young_present': 0.0,\n",
       "  'count': '1',\n",
       "  'season': 'CDB_S1',\n",
       "  'subject_id': 32986502,\n",
       "  'interacting': 0.0},\n",
       " {'dataset': 'snapshot_camdeboo',\n",
       "  'seq_id': 'CDB_S1#B04#1#63',\n",
       "  'location': 'B04',\n",
       "  'images': [{'id': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0153',\n",
       "    'frame_num': 1,\n",
       "    'file': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0153.JPG'},\n",
       "   {'id': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0154',\n",
       "    'frame_num': 2,\n",
       "    'file': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0154.JPG'},\n",
       "   {'id': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0155',\n",
       "    'frame_num': 3,\n",
       "    'file': 'CDB_S1/B04/B04_R1/CDB_S1_B04_R1_IMAG0155.JPG'}],\n",
       "  'class': ['monkeyvervet'],\n",
       "  'resting': 0.2,\n",
       "  'standing': 0.4,\n",
       "  'moving': 0.6,\n",
       "  'young_present': 0.0,\n",
       "  'count': '1',\n",
       "  'season': 'CDB_S1',\n",
       "  'subject_id': 32983267,\n",
       "  'interacting': 0.0}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_camdeboo_megadb.json') as f:\n",
    "    db = json.load(f)\n",
    "\n",
    "len(db)\n",
    "sample(db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "exclusive-satisfaction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7193"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "moving-contract",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16864"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'CDB/CDB_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "appreciated-amateur",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CDB/CDB_public/CDB_S1/A05/A05_R1/CDB_S1_A05_R1_IMAG0112.JPG\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "atomic-peter",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_camdeboo_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "expanded-vertical",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 158 ms, sys: 177 ms, total: 334 ms\n",
      "Wall time: 2.84 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/CDB/CDB_public/CDB_S1/F03/F03_R2/CDB_S1_F03_R2_IMAG0752.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_camdeboo.seqCDB_S1#F03#2#280.frame1.jpg')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_camdeboo'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "smooth-insured",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 29.7 s, sys: 1min 45s, total: 2min 15s\n",
      "Wall time: 2min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "threaded-benjamin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16864"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bronze-palestine",
   "metadata": {},
   "source": [
    "## Snapshot Mountain Zebra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "selective-chamber",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71178"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_mountain_zebra',\n",
       "  'seq_id': 'MTZ_S1#E05#1#4004',\n",
       "  'location': 'E05',\n",
       "  'images': [{'id': 'MTZ_S1/E05/E05_R1/MTZ_S1_E05_R1_IMAG4004',\n",
       "    'frame_num': 1,\n",
       "    'file': 'MTZ_S1/E05/E05_R1/MTZ_S1_E05_R1_IMAG4004.JPG'}],\n",
       "  'class': ['mongooseyellow'],\n",
       "  'datetime': '2017-09-27 15:35:50',\n",
       "  'resting': 0.0,\n",
       "  'standing': 0.33,\n",
       "  'moving': 0.67,\n",
       "  'young_present': 0.0,\n",
       "  'count': '1',\n",
       "  'season': 'MTZ_S1',\n",
       "  'subject_id': 29975409,\n",
       "  'interacting': 0.0},\n",
       " {'dataset': 'snapshot_mountain_zebra',\n",
       "  'seq_id': 'MTZ_S1#B05#3#7111',\n",
       "  'location': 'B05',\n",
       "  'images': [{'id': 'MTZ_S1/B05/B05_R3/MTZ_S1_B05_R3_IMAG7111',\n",
       "    'frame_num': 1,\n",
       "    'file': 'MTZ_S1/B05/B05_R3/MTZ_S1_B05_R3_IMAG7111.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2018-04-29 20:01:08',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'MTZ_S1',\n",
       "  'subject_id': 32757448,\n",
       "  'interacting': None}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_mountain_zebra_megadb.json') as f:\n",
    "    db = json.load(f)\n",
    "\n",
    "len(db)\n",
    "sample(db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "signed-democracy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5593"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "second-discount",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5919"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'MTZ/MTZ_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "pleasant-prague",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'MTZ/MTZ_public/MTZ_S1/B04/B04_R1/MTZ_S1_B04_R1_IMAG0256.JPG\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "early-exemption",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_mountain_zebra_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "resistant-moscow",
   "metadata": {},
   "source": [
    "Short sequences..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "altered-fitting",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47.4 ms, sys: 76.2 ms, total: 124 ms\n",
      "Wall time: 1.15 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/MTZ/MTZ_public/MTZ_S1/G04/G04_R1/MTZ_S1_G04_R1_IMAG3027.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_mountain_zebra.seqMTZ_S1#G04#1#2991.frame1.jpg')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_mountain_zebra'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "transparent-provision",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_pairs[3000:3100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "structured-parker",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13 s, sys: 45 s, total: 58 s\n",
      "Wall time: 55 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "vietnamese-highway",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "efficient-sweet",
   "metadata": {},
   "source": [
    "## Snapshot Kruger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "nonprofit-kitty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4568"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'snapshot_kruger',\n",
       "  'seq_id': 'KRU_S1#8#1#489',\n",
       "  'location': '8',\n",
       "  'images': [{'id': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1401',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1401.JPG'},\n",
       "   {'id': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1402',\n",
       "    'frame_num': 2,\n",
       "    'file': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1402.JPG'},\n",
       "   {'id': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1403',\n",
       "    'frame_num': 3,\n",
       "    'file': 'KRU_S1/8/8_R1/KRU_S1_8_R1_IMAG1403.JPG'}],\n",
       "  'class': ['empty'],\n",
       "  'datetime': '2018-06-28 13:13:30',\n",
       "  'resting': None,\n",
       "  'standing': None,\n",
       "  'moving': None,\n",
       "  'young_present': None,\n",
       "  'count': None,\n",
       "  'season': 'KRU_S1',\n",
       "  'subject_id': 31940127,\n",
       "  'interacting': None},\n",
       " {'dataset': 'snapshot_kruger',\n",
       "  'seq_id': 'KRU_S1#5#1#38',\n",
       "  'location': '5',\n",
       "  'images': [{'id': 'KRU_S1/5/5_R1/KRU_S1_5_R1_IMAG0060',\n",
       "    'frame_num': 1,\n",
       "    'file': 'KRU_S1/5/5_R1/KRU_S1_5_R1_IMAG0060.JPG'}],\n",
       "  'class': ['impala'],\n",
       "  'datetime': '2018-06-03 19:02:16',\n",
       "  'resting': 0.1,\n",
       "  'standing': 0.7,\n",
       "  'moving': 0.7,\n",
       "  'young_present': 0.0,\n",
       "  'count': '4',\n",
       "  'season': 'KRU_S1',\n",
       "  'subject_id': 31937631,\n",
       "  'interacting': 0.0}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/annotation_prep/snapshot_kruger_megadb.json') as f:\n",
    "    db = json.load(f)\n",
    "\n",
    "len(db)\n",
    "sample(db, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "juvenile-sector",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1654"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_seqs = [seq for seq in db if seq['class'][0] != 'empty']\n",
    "len(non_empty_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "associate-holmes",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3540"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download = []\n",
    "\n",
    "path_prefix = 'KRU/KRU_public'\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    for im in seq['images']:\n",
    "        list_to_download.append(os.path.join(path_prefix, im['file']) + '\\n')\n",
    "len(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "official-stadium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KRU/KRU_public/KRU_S1/1/1_R1/KRU_S1_1_R1_IMAG0125.JPG\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_to_download[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "occasional-flooring",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/mink_disk_0/camtraps/megadetectorv5_annotation_prep/batch_12_lists/snapshot_kruger_files.txt', 'w') as f:\n",
    "    f.writelines(list_to_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cardiovascular-criminal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47.8 ms, sys: 22.6 ms, total: 70.3 ms\n",
      "Wall time: 590 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/snapshot-safari/KRU/KRU_public/KRU_S1/9/9_R1/KRU_S1_9_R1_IMAG0065.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12f/snapshot_kruger.seqKRU_S1#9#1#23.frame2.jpg')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset_name = 'snapshot_kruger'\n",
    "container_root = '/mink_disk_0/camtraps/snapshot-safari'\n",
    "\n",
    "path_pairs = []\n",
    "\n",
    "for seq in non_empty_seqs:\n",
    "    seq_id = seq['seq_id']\n",
    "    for im in seq['images']:\n",
    "    \n",
    "        src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "        assert os.path.exists(src_path)\n",
    "\n",
    "        frame = im['frame_num']\n",
    "\n",
    "        dst_path = os.path.join('/mink_disk_0/camtraps/imerit12f', \n",
    "                                f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "        path_pairs.append((src_path, dst_path))\n",
    "\n",
    "len(path_pairs)\n",
    "path_pairs[-50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "severe-volunteer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15.2 s, sys: 53.8 s, total: 1min 8s\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(16) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rental-editing",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cameratraps",
   "language": "python",
   "name": "conda-env-cameratraps-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
